#!/usr/bin/env python
# Create: 2018/11/23
__author__ = '749B'

import json
import logging
import os

from django.utils.dateparse import parse_datetime

logger = logging.getLogger(__name__)


def filenames():
    """遍历文件夹下所有的文件
    返回 filename, hostname
    filename: 文件的完整路径
    hostname: 上级文件夹的名称，就是主机名
    """
    base_folder = os.path.dirname(os.path.split(__file__)[0])
    folder_path = os.path.join(base_folder, 'download')
    # print(folder_path)
    for parent, dirs, files in os.walk(folder_path):
        if files:
            file_list = [os.path.join(parent, file) for file in files]
            for filename in file_list:
                hostname = os.path.dirname(os.path.relpath(filename, folder_path))  # 文件夹名就是主机名
                yield filename, hostname


def parse_json(filename, hostname=None):
    with open(filename, encoding='utf-8') as f:
        for line in f:
            dic = json.loads(line)
            dic['time'] = parse_datetime(dic.pop('@timestamp'))  # 可以直接解析RFC3339
            dic['process'] = dic.pop('syslog-tag')
            if hostname:
                dic['host'] = hostname
            yield dic


def bulk_create(iterator, target_model, target_meta):
    # 对迭代器进行处理，生成一个列表。没有必要继续用迭代器来节省空间。
    # bulk_create内部也是会先把第一个参数转成列表再处理的：objs = list(objs)
    obj_list = [target_model(**obj) for obj in iterator]
    obj_count = len(obj_list)
    logger.info("批量创建 %s，日志记录数： %s" % (target_meta.verbose_name, obj_count))
    target_model.objects.bulk_create(obj_list)


def main(init=False):
    import django
    django.setup()
    from log_audit import models

    end = '_db'
    rename_flag = True
    skip = True
    if init == 1:
        skip = False

    for filename, hostname in filenames():
        if filename.endswith(end):
            # 处理过的文件，做过标记，这里判断后跳过
            if skip:
                continue
            rename_flag = False
        print("解析文件:", filename, hostname)
        # 第三个变量因为是私有属性，到了函数里就别人为是外部引用了，虽然实际没啥影响，但是有错误提示
        bulk_create(parse_json(filename, hostname), models.ParsedLog, models.ParsedLog._meta)
        if rename_flag:
            os.rename(filename, filename + end)
        else:
            rename_flag = True
    print("任务完成")







